# -*- coding: utf-8 -*-

BOT_NAME = 'first_scrapy'

SPIDER_MODULES = ['first_scrapy.spiders']
NEWSPIDER_MODULE = 'first_scrapy.spiders'

#不遵守robot.txt协议
ROBOTSTXT_OBEY = False

#设置一下默认的header信息
DEFAULT_REQUEST_HEADERS = {
    'User-Agent' : 'Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/69.0.3497.100 Safari/537.36',
    'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8',
    'Accept-Language': 'zh-CN,zh;q=0.9,en;q=0.8',
}

#开启自定义的管道
ITEM_PIPELINES = {
   'first_scrapy.pipelines.FirstScrapyPipeline': 300,
}
